#coding:utf-8
from scrapy.http import Request
from scrapy.spiders import Spider

from fetchGit.items import RepertoryZipedItem
from fetchGit.Roles import roles


class GitRepSpider(Spider):
    name = 'gitRepSpider'
    url = 'https://github.com/{}?tab=repositories'
    start_urls = [url.format(u) for u in roles]

    def parse(self, response):
        next_url = response.xpath('//a[@class="next_page"]/@href').extract()
        if len(next_url) > 0:
            url = response.urljoin(next_url[0])
            yield Request(url, callback=self.parse)
        projs = response.xpath(
            '//a[@itemprop="name codeRepository"]/@href').extract()
        proj_url_format = 'https://github.com/{}/archive/master.zip'
        proj_urls = [(proj_url_format.format(p.strip()), p) for p in projs]
        for proj_url, item_data in proj_urls:
            item = RepertoryZipedItem()
            item_partial = item_data.strip().split('/')
            item['fileName'] = item_partial[2]
            item['auth'] = item_partial[1]
            yield Request(proj_url, callback=self.down_parse, meta={'rep': item})

    def down_parse(self, response):  #下载数据
        item = response.meta['rep']
        item['data'] = response.body
        return item